# -*- coding: utf-8 -*-
import scrapy
from tutorial.items import TutorialItem

class QuotesSpider(scrapy.Spider):
    name = 'quotes'
    allowed_domains = ['quotes.toscrape.com']
    start_urls = ['http://quotes.toscrape.com/']

    """
    基于管道的持久化
    需要打开settings.py中的
    ITEM_PIPELINES = {
    'tutorial.pipelines.TutorialPipeline': 300,
    }
    还要在items.py中设置值
    piplines.py中要有具体的操作步骤
    """
    def parse(self, response):
        quotes =  response.xpath('//div/div[@class="quote"]')
        for quote in quotes:
            item = TutorialItem()
            item['text'] = quote.xpath('./span[@class="text"]/text()').extract_first()
            item['author'] = quote.xpath('./span/small/text()').extract_first()
            item['tags'] = quote.xpath('./div/a/text()').extract()
            yield item
        next = response.xpath('/html/body/div/div[2]/div[1]/nav/ul/li[@class="next"]/a/@href').extract_first()
        url = response.urljoin(next)
        yield scrapy.Request(url = url,callback=self.parse)

        """以下是基于终端指令的持久化"""
        # def parse(self, response):
        # quotes =  response.xpath('//div/div[@class="quote"]')
        # all_data = []
        # for quote in quotes:
        #     text = quote.xpath('./span[@class="text"]/text()').extract_first()
        #     author = quote.xpath('./span/small/text()').extract_first()
        #     tags = quote.xpath('./div/a/text()').extract()
        #     dic = {
        #         "text":text,
        #         "author":author,
        #         "tags":tags
        #     }
        #     all_data.append(dic)
        # return all_data
